<!DOCTYPE HTML>
<html lang="en" >
    <!-- Start book Python数据分析课程讲义 -->
    <head>
        <!-- head:start -->
        <meta charset="UTF-8">
        <meta http-equiv="X-UA-Compatible" content="IE=edge" />
        <title>NLTK与自然语言处理基础 | Python数据分析课程讲义</title>
        <meta content="text/html; charset=utf-8" http-equiv="Content-Type">
        <meta name="description" content="">
        <meta name="generator" content="GitBook 2.6.7">
        <meta name="author" content="BigCat">
        
        <meta name="HandheldFriendly" content="true"/>
        <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
        <meta name="apple-mobile-web-app-capable" content="yes">
        <meta name="apple-mobile-web-app-status-bar-style" content="black">
        <link rel="apple-touch-icon-precomposed" sizes="152x152" href="../../gitbook/images/apple-touch-icon-precomposed-152.png">
        <link rel="shortcut icon" href="../../gitbook/images/favicon.ico" type="image/x-icon">
        
    <link rel="stylesheet" href="../../gitbook/style.css">
    
        
        <link rel="stylesheet" href="../../gitbook/plugins/gitbook-plugin-tbfed-pagefooter/footer.css">
        
    
        
        <link rel="stylesheet" href="../../gitbook/plugins/gitbook-plugin-splitter/splitter.css">
        
    
        
        <link rel="stylesheet" href="../../gitbook/plugins/gitbook-plugin-toggle-chapters/toggle.css">
        
    
        
        <link rel="stylesheet" href="../../gitbook/plugins/gitbook-plugin-highlight/website.css">
        
    
        
        <link rel="stylesheet" href="../../gitbook/plugins/gitbook-plugin-fontsettings/website.css">
        
    
    

        
    
    
    <link rel="next" href="../../file/part06/6.2.html" />
    
    
    <link rel="prev" href="../../file/part06/6.html" />
    

        <!-- head:end -->
    </head>
    <body>
        <!-- body:start -->
        
    <div class="book"
        data-level="5.1"
        data-chapter-title="NLTK与自然语言处理基础"
        data-filepath="file/part06/6.1.md"
        data-basepath="../.."
        data-revision="Thu Apr 27 2017 00:50:19 GMT+0800 (CST)"
        data-innerlanguage="">
    

<div class="book-summary">
    <nav role="navigation">
        <ul class="summary">
            
            
            
            

            

            
    
        <li class="chapter " data-level="0" data-path="index.html">
            
                
                    <a href="../../index.html">
                
                        <i class="fa fa-check"></i>
                        
                        传智播客Python学院数据分析
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1" data-path="file/part01/1.html">
            
                
                    <a href="../../file/part01/1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.</b>
                        
                        一、工作环境准备及数据分析建模理论基础
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.1" data-path="file/part01/1.1.html">
            
                
                    <a href="../../file/part01/1.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.1.</b>
                        
                        Python 3.x新特性和编码回顾
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1.2" data-path="file/part01/1.2.html">
            
                
                    <a href="../../file/part01/1.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.2.</b>
                        
                        DIKW模型与数据工程
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1.3" data-path="file/part01/1.3.html">
            
                
                    <a href="../../file/part01/1.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.3.</b>
                        
                        数据分析建模理论基础
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2" data-path="file/part02/2.html">
            
                
                    <a href="../../file/part02/2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.</b>
                        
                        二、科学计算工具NumPy
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.1" data-path="file/part02/2.1.html">
            
                
                    <a href="../../file/part02/2.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.1.</b>
                        
                        ndarray的创建与数据类型
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.2" data-path="file/part02/2.2.html">
            
                
                    <a href="../../file/part02/2.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.2.</b>
                        
                        ndarray的矩阵处理
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.3" data-path="file/part02/2.3.html">
            
                
                    <a href="../../file/part02/2.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.3.</b>
                        
                        ndarray的元素处理
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.4" data-path="file/part02/2.4.html">
            
                
                    <a href="../../file/part02/2.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.4.</b>
                        
                        实战案例：2016美国总统大选民意调查统计
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="3" data-path="file/part03/3.html">
            
                
                    <a href="../../file/part03/3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.</b>
                        
                        三、数据分析工具Pandas
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="3.1" data-path="file/part03/3.1.html">
            
                
                    <a href="../../file/part03/3.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.1.</b>
                        
                        Pandas的数据结构
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.2" data-path="file/part03/3.2.html">
            
                
                    <a href="../../file/part03/3.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.2.</b>
                        
                        Pandas的索引操作
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.3" data-path="file/part03/3.3.html">
            
                
                    <a href="../../file/part03/3.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.3.</b>
                        
                        Pandas的对齐运算
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.4" data-path="file/part03/3.4.html">
            
                
                    <a href="../../file/part03/3.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.4.</b>
                        
                        Pandas的函数应用
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.5" data-path="file/part03/3.5.html">
            
                
                    <a href="../../file/part03/3.5.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.5.</b>
                        
                        Pandas的层级索引
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.6" data-path="file/part03/3.6.html">
            
                
                    <a href="../../file/part03/3.6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.6.</b>
                        
                        Pandas统计计算和描述
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.7" data-path="file/part03/3.7.html">
            
                
                    <a href="../../file/part03/3.7.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.7.</b>
                        
                        Pandas分组与聚合
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.8" data-path="file/part03/3.8.html">
            
                
                    <a href="../../file/part03/3.8.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.8.</b>
                        
                        数据清洗、合并、转化和重构
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.9" data-path="file/part03/3.9.html">
            
                
                    <a href="../../file/part03/3.9.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.9.</b>
                        
                        聚类模型 -- K-Means介绍
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.10" data-path="file/part03/3.10.html">
            
                
                    <a href="../../file/part03/3.10.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.10.</b>
                        
                        实战案例：全球食品数据分析
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="4" data-path="file/part04/4.html">
            
                
                    <a href="../../file/part04/4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.</b>
                        
                        四、数据可视化工具
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="4.1" data-path="file/part04/4.1.html">
            
                
                    <a href="../../file/part04/4.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.1.</b>
                        
                        Matplotlib绘图
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.2" data-path="file/part04/4.2.html">
            
                
                    <a href="../../file/part04/4.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.2.</b>
                        
                        Seaborn绘图
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.3" data-path="file/part04/4.3.html">
            
                
                    <a href="../../file/part04/4.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.3.</b>
                        
                        Bokeh绘图
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.4" data-path="file/part04/4.4.html">
            
                
                    <a href="../../file/part04/4.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.4.</b>
                        
                        实战案例：世界高峰数据可视化
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="5" data-path="file/part06/6.html">
            
                
                    <a href="../../file/part06/6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.</b>
                        
                        五、自然语言处理NLTK
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter active" data-level="5.1" data-path="file/part06/6.1.html">
            
                
                    <a href="../../file/part06/6.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.1.</b>
                        
                        NLTK与自然语言处理基础
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="5.2" data-path="file/part06/6.2.html">
            
                
                    <a href="../../file/part06/6.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.2.</b>
                        
                        jieba分词
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="5.3" data-path="file/part06/6.3.html">
            
                
                    <a href="../../file/part06/6.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.3.</b>
                        
                        情感分析
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="5.4" data-path="file/part06/6.4.html">
            
                
                    <a href="../../file/part06/6.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.4.</b>
                        
                        文本相似度和分类
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="5.5" data-path="file/part06/6.6.html">
            
                
                    <a href="../../file/part06/6.6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.5.</b>
                        
                        实战案例：微博情感分析
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    


            
            <li class="divider"></li>
            <li>
                <a href="https://www.gitbook.com" target="blank" class="gitbook-link">
                    Published with GitBook
                </a>
            </li>
            
        </ul>
    </nav>
</div>

    <div class="book-body">
        <div class="body-inner">
            <div class="book-header" role="navigation">
    <!-- Actions Left -->
    

    <!-- Title -->
    <h1>
        <i class="fa fa-circle-o-notch fa-spin"></i>
        <a href="../../" >Python数据分析课程讲义</a>
    </h1>
</div>

            <div class="page-wrapper" tabindex="-1" role="main">
                <div class="page-inner">
                
                
                    <section class="normal" id="section-">
                    
                        <h2 id="nltk-natural-language-toolkit">NLTK (Natural Language Toolkit)</h2>
<p>NTLK&#x662F;&#x8457;&#x540D;&#x7684;Python&#x81EA;&#x7136;&#x8BED;&#x8A00;&#x5904;&#x7406;&#x5DE5;&#x5177;&#x5305;&#xFF0C;&#x4F46;&#x662F;&#x4E3B;&#x8981;&#x9488;&#x5BF9;&#x7684;&#x662F;&#x82F1;&#x6587;&#x5904;&#x7406;&#x3002;NLTK&#x914D;&#x5957;&#x6709;&#x6587;&#x6863;&#xFF0C;&#x6709;&#x8BED;&#x6599;&#x5E93;&#xFF0C;&#x6709;&#x4E66;&#x7C4D;&#x3002;</p>
<ul>
<li>NLP&#x9886;&#x57DF;&#x4E2D;&#x6700;&#x5E38;&#x7528;&#x7684;&#x4E00;&#x4E2A;Python&#x5E93;</li>
<li>&#x5F00;&#x6E90;&#x9879;&#x76EE;</li>
<li>&#x81EA;&#x5E26;&#x5206;&#x7C7B;&#x3001;&#x5206;&#x8BCD;&#x7B49;&#x529F;&#x80FD;</li>
<li>&#x5F3A;&#x5927;&#x7684;&#x793E;&#x533A;&#x652F;&#x6301;</li>
<li>&#x8BED;&#x6599;&#x5E93;&#xFF0C;&#x8BED;&#x8A00;&#x7684;&#x5B9E;&#x9645;&#x4F7F;&#x7528;&#x4E2D;&#x771F;&#x662F;&#x51FA;&#x73B0;&#x8FC7;&#x7684;&#x8BED;&#x8A00;&#x6750;&#x6599;</li>
<li><a href="http://www.nltk.org/py-modindex.html" target="_blank">http://www.nltk.org/py-modindex.html</a></li>
</ul>
<p>&#x5728;NLTK&#x7684;&#x4E3B;&#x9875;&#x8BE6;&#x7EC6;&#x4ECB;&#x7ECD;&#x4E86;&#x5982;&#x4F55;&#x5728;Mac&#x3001;Linux&#x548C;Windows&#x4E0B;&#x5B89;&#x88C5;NLTK&#xFF1A;<a href="http://nltk.org/install.html" target="_blank">http://nltk.org/install.html</a> &#xFF0C;&#x5EFA;&#x8BAE;&#x76F4;&#x63A5;&#x4E0B;&#x8F7D;Anaconda&#xFF0C;&#x7701;&#x53BB;&#x4E86;&#x5927;&#x90E8;&#x5206;&#x5305;&#x7684;&#x5B89;&#x88C5;&#xFF0C;&#x5B89;&#x88C5;NLTK&#x5B8C;&#x6BD5;&#xFF0C;&#x53EF;&#x4EE5;import nltk&#x6D4B;&#x8BD5;&#x4E00;&#x4E0B;&#xFF0C;&#x5982;&#x679C;&#x6CA1;&#x6709;&#x95EE;&#x9898;&#xFF0C;&#x8FD8;&#x6709;&#x4E0B;&#x8F7D;NLTK&#x5B98;&#x65B9;&#x63D0;&#x4F9B;&#x7684;&#x76F8;&#x5173;&#x8BED;&#x6599;&#x3002;</p>
<h2 id="&#x5B89;&#x88C5;&#x6B65;&#x9AA4;&#xFF1A;">&#x5B89;&#x88C5;&#x6B65;&#x9AA4;&#xFF1A;</h2>
<ol>
<li><p>&#x4E0B;&#x8F7D;NLTK&#x5305;
 <code>pip install nltk</code></p>
</li>
<li><p>&#x8FD0;&#x884C;Python&#xFF0C;&#x5E76;&#x8F93;&#x5165;&#x4E0B;&#x9762;&#x7684;&#x6307;&#x4EE4;</p>
<pre><code class="lang-python"> <span class="hljs-keyword">import</span> nltk
 nltk.download()
</code></pre>
</li>
<li><p>&#x5F39;&#x51FA;&#x4E0B;&#x9762;&#x7684;&#x7A97;&#x53E3;&#xFF0C;&#x5EFA;&#x8BAE;&#x5B89;&#x88C5;&#x6240;&#x6709;&#x7684;&#x5305; &#xFF0C;&#x5373;<code>all</code></p>
<p> <img src="../images/nltk_install.png" alt=""></p>
</li>
<li><p>&#x6D4B;&#x8BD5;&#x4F7F;&#x7528;&#xFF1A;</p>
<p> <img src="../images/nltk_test.png" alt=""></p>
</li>
</ol>
<h2 id="&#x8BED;&#x6599;&#x5E93;">&#x8BED;&#x6599;&#x5E93;</h2>
<blockquote>
<p><code>nltk.corpus</code></p>
</blockquote>
<pre><code class="lang-python"><span class="hljs-keyword">import</span> nltk
<span class="hljs-keyword">from</span> nltk.corpus <span class="hljs-keyword">import</span> brown <span class="hljs-comment"># &#x9700;&#x8981;&#x4E0B;&#x8F7D;brown&#x8BED;&#x6599;&#x5E93;</span>
<span class="hljs-comment"># &#x5F15;&#x7528;&#x5E03;&#x6717;&#x5927;&#x5B66;&#x7684;&#x8BED;&#x6599;&#x5E93;</span>

<span class="hljs-comment"># &#x67E5;&#x770B;&#x8BED;&#x6599;&#x5E93;&#x5305;&#x542B;&#x7684;&#x7C7B;&#x522B;</span>
print(brown.categories())

<span class="hljs-comment"># &#x67E5;&#x770B;brown&#x8BED;&#x6599;&#x5E93;</span>
print(<span class="hljs-string">&apos;&#x5171;&#x6709;{}&#x4E2A;&#x53E5;&#x5B50;&apos;</span>.format(len(brown.sents())))
print(<span class="hljs-string">&apos;&#x5171;&#x6709;{}&#x4E2A;&#x5355;&#x8BCD;&apos;</span>.format(len(brown.words())))
</code></pre>
<p>&#x6267;&#x884C;&#x7ED3;&#x679C;&#xFF1A;</p>
<pre><code class="lang-python">[<span class="hljs-string">&apos;adventure&apos;</span>, <span class="hljs-string">&apos;belles_lettres&apos;</span>, <span class="hljs-string">&apos;editorial&apos;</span>, <span class="hljs-string">&apos;fiction&apos;</span>, <span class="hljs-string">&apos;government&apos;</span>, <span class="hljs-string">&apos;hobbies&apos;</span>, <span class="hljs-string">&apos;humor&apos;</span>, <span class="hljs-string">&apos;learned&apos;</span>, <span class="hljs-string">&apos;lore&apos;</span>, <span class="hljs-string">&apos;mystery&apos;</span>, <span class="hljs-string">&apos;news&apos;</span>, <span class="hljs-string">&apos;religion&apos;</span>, <span class="hljs-string">&apos;reviews&apos;</span>, <span class="hljs-string">&apos;romance&apos;</span>, <span class="hljs-string">&apos;science_fiction&apos;</span>]

&#x5171;&#x6709;<span class="hljs-number">57340</span>&#x4E2A;&#x53E5;&#x5B50;
&#x5171;&#x6709;<span class="hljs-number">1161192</span>&#x4E2A;&#x5355;&#x8BCD;
</code></pre>
<h2 id="&#x5206;&#x8BCD;-tokenize">&#x5206;&#x8BCD; (tokenize)</h2>
<ul>
<li>&#x5C06;&#x53E5;&#x5B50;&#x62C6;&#x5206;&#x6210;&#x5177;&#x6709;&#x8BED;&#x8A00;&#x8BED;&#x4E49;&#x5B66;&#x4E0A;&#x610F;&#x4E49;&#x7684;&#x8BCD;</li>
<li>&#x4E2D;&#x3001;&#x82F1;&#x6587;&#x5206;&#x8BCD;&#x533A;&#x522B;&#xFF1A;<ul>
<li>&#x82F1;&#x6587;&#x4E2D;&#xFF0C;&#x5355;&#x8BCD;&#x4E4B;&#x95F4;&#x662F;&#x4EE5;&#x7A7A;&#x683C;&#x4F5C;&#x4E3A;&#x81EA;&#x7136;&#x5206;&#x754C;&#x7B26;&#x7684;</li>
<li>&#x4E2D;&#x6587;&#x4E2D;&#x6CA1;&#x6709;&#x4E00;&#x4E2A;&#x5F62;&#x5F0F;&#x4E0A;&#x7684;&#x5206;&#x754C;&#x7B26;&#xFF0C;&#x5206;&#x8BCD;&#x6BD4;&#x82F1;&#x6587;&#x590D;&#x6742;&#x7684;&#x591A;</li>
</ul>
</li>
<li>&#x4E2D;&#x6587;&#x5206;&#x8BCD;&#x5DE5;&#x5177;&#xFF0C;&#x5982;&#xFF1A;&#x7ED3;&#x5DF4;&#x5206;&#x8BCD; <code>pip install jieba</code></li>
<li>&#x5F97;&#x5230;&#x5206;&#x8BCD;&#x7ED3;&#x679C;&#x540E;&#xFF0C;&#x4E2D;&#x82F1;&#x6587;&#x7684;&#x540E;&#x7EED;&#x5904;&#x7406;&#x6CA1;&#x6709;&#x592A;&#x5927;&#x533A;&#x522B;</li>
</ul>
<pre><code class="lang-python"><span class="hljs-comment"># &#x5BFC;&#x5165;jieba&#x5206;&#x8BCD;</span>
<span class="hljs-keyword">import</span> jieba

seg_list = jieba.cut(<span class="hljs-string">&quot;&#x6B22;&#x8FCE;&#x6765;&#x5230;&#x9ED1;&#x9A6C;&#x7A0B;&#x5E8F;&#x5458;Python&#x5B66;&#x79D1;&quot;</span>, cut_all=<span class="hljs-keyword">True</span>)
print(<span class="hljs-string">&quot;&#x5168;&#x6A21;&#x5F0F;: &quot;</span> + <span class="hljs-string">&quot;/ &quot;</span>.join(seg_list))  <span class="hljs-comment"># &#x5168;&#x6A21;&#x5F0F;</span>

seg_list = jieba.cut(<span class="hljs-string">&quot;&#x6B22;&#x8FCE;&#x6765;&#x5230;&#x9ED1;&#x9A6C;&#x7A0B;&#x5E8F;&#x5458;Python&#x5B66;&#x79D1;&quot;</span>, cut_all=<span class="hljs-keyword">False</span>)
print(<span class="hljs-string">&quot;&#x7CBE;&#x786E;&#x6A21;&#x5F0F;: &quot;</span> + <span class="hljs-string">&quot;/ &quot;</span>.join(seg_list))  <span class="hljs-comment"># &#x7CBE;&#x786E;&#x6A21;&#x5F0F;</span>
</code></pre>
<p>&#x8FD0;&#x884C;&#x7ED3;&#x679C;&#xFF1A;</p>
<pre><code>&#x5168;&#x6A21;&#x5F0F;: &#x6B22;&#x8FCE;/ &#x8FCE;&#x6765;/ &#x6765;&#x5230;/ &#x9ED1;&#x9A6C;/ &#x7A0B;&#x5E8F;/ &#x7A0B;&#x5E8F;&#x5458;/ Python/ &#x5B66;&#x79D1;
&#x7CBE;&#x786E;&#x6A21;&#x5F0F;: &#x6B22;&#x8FCE;/ &#x6765;&#x5230;/ &#x9ED1;&#x9A6C;/ &#x7A0B;&#x5E8F;&#x5458;/ Python/ &#x5B66;&#x79D1;
</code></pre><h2 id="&#x8BCD;&#x5F62;&#x95EE;&#x9898;">&#x8BCD;&#x5F62;&#x95EE;&#x9898;</h2>
<ul>
<li>look, looked, looking</li>
<li>&#x5F71;&#x54CD;&#x8BED;&#x6599;&#x5B66;&#x4E60;&#x7684;&#x51C6;&#x786E;&#x5EA6;</li>
<li>&#x8BCD;&#x5F62;&#x5F52;&#x4E00;&#x5316;</li>
</ul>
<h4 id="1-&#x8BCD;&#x5E72;&#x63D0;&#x53D6;stemming">1. &#x8BCD;&#x5E72;&#x63D0;&#x53D6;(stemming)</h4>
<p>&#x793A;&#x4F8B;&#xFF1A;</p>
<pre><code class="lang-python"><span class="hljs-comment"># PorterStemmer</span>
<span class="hljs-keyword">from</span> nltk.stem.porter <span class="hljs-keyword">import</span> PorterStemmer

porter_stemmer = PorterStemmer()
print(porter_stemmer.stem(<span class="hljs-string">&apos;looked&apos;</span>))
print(porter_stemmer.stem(<span class="hljs-string">&apos;looking&apos;</span>))

<span class="hljs-comment"># &#x8FD0;&#x884C;&#x7ED3;&#x679C;&#xFF1A;</span>
<span class="hljs-comment"># look</span>
<span class="hljs-comment"># look</span>
</code></pre>
<p>&#x793A;&#x4F8B;&#xFF1A;</p>
<pre><code class="lang-python"><span class="hljs-comment"># SnowballStemmer</span>
<span class="hljs-keyword">from</span> nltk.stem <span class="hljs-keyword">import</span> SnowballStemmer

snowball_stemmer = SnowballStemmer(<span class="hljs-string">&apos;english&apos;</span>)
print(snowball_stemmer.stem(<span class="hljs-string">&apos;looked&apos;</span>))
print(snowball_stemmer.stem(<span class="hljs-string">&apos;looking&apos;</span>))

<span class="hljs-comment"># &#x8FD0;&#x884C;&#x7ED3;&#x679C;&#xFF1A;</span>
<span class="hljs-comment"># look</span>
<span class="hljs-comment"># look</span>
</code></pre>
<p>&#x793A;&#x4F8B;&#xFF1A;</p>
<pre><code class="lang-python"><span class="hljs-comment"># LancasterStemmer</span>
<span class="hljs-keyword">from</span> nltk.stem.lancaster <span class="hljs-keyword">import</span> LancasterStemmer

lancaster_stemmer = LancasterStemmer()
print(lancaster_stemmer.stem(<span class="hljs-string">&apos;looked&apos;</span>))
print(lancaster_stemmer.stem(<span class="hljs-string">&apos;looking&apos;</span>))

<span class="hljs-comment"># &#x8FD0;&#x884C;&#x7ED3;&#x679C;&#xFF1A;</span>
<span class="hljs-comment"># look</span>
<span class="hljs-comment"># look</span>
</code></pre>
<h4 id="2-&#x8BCD;&#x5F62;&#x5F52;&#x5E76;lemmatization">2. &#x8BCD;&#x5F62;&#x5F52;&#x5E76;(lemmatization)</h4>
<ul>
<li><p>stemming&#xFF0C;&#x8BCD;&#x5E72;&#x63D0;&#x53D6;&#xFF0C;&#x5982;&#x5C06;ing, ed&#x53BB;&#x6389;&#xFF0C;&#x53EA;&#x4FDD;&#x7559;&#x5355;&#x8BCD;&#x4E3B;&#x5E72;</p>
</li>
<li><p>lemmatization&#xFF0C;&#x8BCD;&#x5F62;&#x5F52;&#x5E76;&#xFF0C;&#x5C06;&#x5355;&#x8BCD;&#x7684;&#x5404;&#x79CD;&#x8BCD;&#x5F62;&#x5F52;&#x5E76;&#x6210;&#x4E00;&#x79CD;&#x5F62;&#x5F0F;&#xFF0C;&#x5982;am, is, are -&gt; be, went-&gt;go</p>
</li>
<li><p>NLTK&#x4E2D;&#x7684;stemmer</p>
<blockquote>
<p>PorterStemmer, SnowballStemmer, LancasterStemmer</p>
</blockquote>
</li>
<li><p>NLTK&#x4E2D;&#x7684;lemma</p>
<blockquote>
<p>WordNetLemmatizer</p>
</blockquote>
</li>
<li><p>&#x95EE;&#x9898;</p>
<blockquote>
<p>went &#x52A8;&#x8BCD; -&gt; go&#xFF0C; &#x8D70; 
Went &#x540D;&#x8BCD; -&gt; Went&#xFF0C;&#x6587;&#x7279;</p>
</blockquote>
</li>
<li><p>&#x6307;&#x660E;&#x8BCD;&#x6027;&#x53EF;&#x4EE5;&#x66F4;&#x51C6;&#x786E;&#x5730;&#x8FDB;&#x884C;lemma</p>
</li>
</ul>
<p>&#x793A;&#x4F8B;&#xFF1A;</p>
<pre><code class="lang-python"><span class="hljs-keyword">from</span> nltk.stem <span class="hljs-keyword">import</span> WordNetLemmatizer 
<span class="hljs-comment"># &#x9700;&#x8981;&#x4E0B;&#x8F7D;wordnet&#x8BED;&#x6599;&#x5E93;</span>

wordnet_lematizer = WordNetLemmatizer()
print(wordnet_lematizer.lemmatize(<span class="hljs-string">&apos;cats&apos;</span>))
print(wordnet_lematizer.lemmatize(<span class="hljs-string">&apos;boxes&apos;</span>))
print(wordnet_lematizer.lemmatize(<span class="hljs-string">&apos;are&apos;</span>))
print(wordnet_lematizer.lemmatize(<span class="hljs-string">&apos;went&apos;</span>))

<span class="hljs-comment"># &#x8FD0;&#x884C;&#x7ED3;&#x679C;&#xFF1A;</span>
<span class="hljs-comment"># cat</span>
<span class="hljs-comment"># box</span>
<span class="hljs-comment"># are</span>
<span class="hljs-comment"># went</span>
</code></pre>
<p>&#x793A;&#x4F8B;&#xFF1A;</p>
<pre><code class="lang-python"><span class="hljs-comment"># &#x6307;&#x660E;&#x8BCD;&#x6027;&#x53EF;&#x4EE5;&#x66F4;&#x51C6;&#x786E;&#x5730;&#x8FDB;&#x884C;lemma</span>
<span class="hljs-comment"># lemmatize &#x9ED8;&#x8BA4;&#x4E3A;&#x540D;&#x8BCD;</span>
print(wordnet_lematizer.lemmatize(<span class="hljs-string">&apos;are&apos;</span>, pos=<span class="hljs-string">&apos;v&apos;</span>))
print(wordnet_lematizer.lemmatize(<span class="hljs-string">&apos;went&apos;</span>, pos=<span class="hljs-string">&apos;v&apos;</span>))

<span class="hljs-comment"># &#x8FD0;&#x884C;&#x7ED3;&#x679C;&#xFF1A;</span>
<span class="hljs-comment"># be</span>
<span class="hljs-comment"># go</span>
</code></pre>
<h4 id="3-&#x8BCD;&#x6027;&#x6807;&#x6CE8;-partofspeech">3. &#x8BCD;&#x6027;&#x6807;&#x6CE8; (Part-Of-Speech)</h4>
<ul>
<li><p>NLTK&#x4E2D;&#x7684;&#x8BCD;&#x6027;&#x6807;&#x6CE8;</p>
<blockquote>
<p>nltk.word_tokenize()</p>
</blockquote>
</li>
</ul>
<p>&#x793A;&#x4F8B;&#xFF1A;</p>
<pre><code class="lang-python"><span class="hljs-keyword">import</span> nltk

words = nltk.word_tokenize(<span class="hljs-string">&apos;Python is a widely used programming language.&apos;</span>)
print(nltk.pos_tag(words)) <span class="hljs-comment"># &#x9700;&#x8981;&#x4E0B;&#x8F7D; averaged_perceptron_tagger</span>

<span class="hljs-comment"># &#x8FD0;&#x884C;&#x7ED3;&#x679C;&#xFF1A;</span>
<span class="hljs-comment"># [(&apos;Python&apos;, &apos;NNP&apos;), (&apos;is&apos;, &apos;VBZ&apos;), (&apos;a&apos;, &apos;DT&apos;), (&apos;widely&apos;, &apos;RB&apos;), (&apos;used&apos;, &apos;VBN&apos;), (&apos;programming&apos;, &apos;NN&apos;), (&apos;language&apos;, &apos;NN&apos;), (&apos;.&apos;, &apos;.&apos;)]</span>
</code></pre>
<h4 id="4-&#x53BB;&#x9664;&#x505C;&#x7528;&#x8BCD;">4. &#x53BB;&#x9664;&#x505C;&#x7528;&#x8BCD;</h4>
<ul>
<li>&#x4E3A;&#x8282;&#x7701;&#x5B58;&#x50A8;&#x7A7A;&#x95F4;&#x548C;&#x63D0;&#x9AD8;&#x641C;&#x7D22;&#x6548;&#x7387;&#xFF0C;NLP&#x4E2D;&#x4F1A;&#x81EA;&#x52A8;&#x8FC7;&#x6EE4;&#x6389;&#x67D0;&#x4E9B;&#x5B57;&#x6216;&#x8BCD;</li>
<li>&#x505C;&#x7528;&#x8BCD;&#x90FD;&#x662F;&#x4EBA;&#x5DE5;&#x8F93;&#x5165;&#x3001;&#x975E;&#x81EA;&#x52A8;&#x5316;&#x751F;&#x6210;&#x7684;&#xFF0C;&#x5F62;&#x6210;&#x505C;&#x7528;&#x8BCD;&#x8868;</li>
<li><p>&#x5206;&#x7C7B;</p>
<blockquote>
<p>&#x8BED;&#x8A00;&#x4E2D;&#x7684;&#x529F;&#x80FD;&#x8BCD;&#xFF0C;&#x5982;the, is&#x2026;</p>
<p>&#x8BCD;&#x6C47;&#x8BCD;&#xFF0C;&#x901A;&#x5E38;&#x662F;&#x4F7F;&#x7528;&#x5E7F;&#x6CDB;&#x7684;&#x8BCD;&#xFF0C;&#x5982;want</p>
</blockquote>
</li>
<li><p>&#x4E2D;&#x6587;&#x505C;&#x7528;&#x8BCD;&#x8868;</p>
<blockquote>
<p>&#x4E2D;&#x6587;&#x505C;&#x7528;&#x8BCD;&#x5E93;</p>
<p>&#x54C8;&#x5DE5;&#x5927;&#x505C;&#x7528;&#x8BCD;&#x8868;</p>
<p>&#x56DB;&#x5DDD;&#x5927;&#x5B66;&#x673A;&#x5668;&#x667A;&#x80FD;&#x5B9E;&#x9A8C;&#x5BA4;&#x505C;&#x7528;&#x8BCD;&#x5E93;</p>
<p>&#x767E;&#x5EA6;&#x505C;&#x7528;&#x8BCD;&#x5217;&#x8868;</p>
</blockquote>
</li>
<li><p>&#x5176;&#x4ED6;&#x8BED;&#x8A00;&#x505C;&#x7528;&#x8BCD;&#x8868;</p>
<blockquote>
<p><a href="http://www.ranks.nl/stopwords" target="_blank">http://www.ranks.nl/stopwords</a></p>
</blockquote>
</li>
<li><p>&#x4F7F;&#x7528;NLTK&#x53BB;&#x9664;&#x505C;&#x7528;&#x8BCD;</p>
<blockquote>
<p>stopwords.words()</p>
</blockquote>
</li>
</ul>
<p>&#x793A;&#x4F8B;&#xFF1A;</p>
<pre><code class="lang-python"><span class="hljs-keyword">from</span> nltk.corpus <span class="hljs-keyword">import</span> stopwords <span class="hljs-comment"># &#x9700;&#x8981;&#x4E0B;&#x8F7D;stopwords</span>

filtered_words = [word <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> words <span class="hljs-keyword">if</span> word <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> stopwords.words(<span class="hljs-string">&apos;english&apos;</span>)]
print(<span class="hljs-string">&apos;&#x539F;&#x59CB;&#x8BCD;&#xFF1A;&apos;</span>, words)
print(<span class="hljs-string">&apos;&#x53BB;&#x9664;&#x505C;&#x7528;&#x8BCD;&#x540E;&#xFF1A;&apos;</span>, filtered_words)

<span class="hljs-comment"># &#x8FD0;&#x884C;&#x7ED3;&#x679C;&#xFF1A;</span>
<span class="hljs-comment"># &#x539F;&#x59CB;&#x8BCD;&#xFF1A; [&apos;Python&apos;, &apos;is&apos;, &apos;a&apos;, &apos;widely&apos;, &apos;used&apos;, &apos;programming&apos;, &apos;language&apos;, &apos;.&apos;]</span>
<span class="hljs-comment"># &#x53BB;&#x9664;&#x505C;&#x7528;&#x8BCD;&#x540E;&#xFF1A; [&apos;Python&apos;, &apos;widely&apos;, &apos;used&apos;, &apos;programming&apos;, &apos;language&apos;, &apos;.&apos;]</span>
</code></pre>
<h4 id="5-&#x5178;&#x578B;&#x7684;&#x6587;&#x672C;&#x9884;&#x5904;&#x7406;&#x6D41;&#x7A0B;">5. &#x5178;&#x578B;&#x7684;&#x6587;&#x672C;&#x9884;&#x5904;&#x7406;&#x6D41;&#x7A0B;</h4>
<p>&#x793A;&#x4F8B;&#xFF1A;</p>
<pre><code class="lang-python"><span class="hljs-keyword">import</span> nltk
<span class="hljs-keyword">from</span> nltk.stem <span class="hljs-keyword">import</span> WordNetLemmatizer
<span class="hljs-keyword">from</span> nltk.corpus <span class="hljs-keyword">import</span> stopwords

<span class="hljs-comment"># &#x539F;&#x59CB;&#x6587;&#x672C;</span>
raw_text = <span class="hljs-string">&apos;Life is like a box of chocolates. You never know what you\&apos;re gonna get.&apos;</span>

<span class="hljs-comment"># &#x5206;&#x8BCD;</span>
raw_words = nltk.word_tokenize(raw_text)

<span class="hljs-comment"># &#x8BCD;&#x5F62;&#x5F52;&#x4E00;&#x5316;</span>
wordnet_lematizer = WordNetLemmatizer()
words = [wordnet_lematizer.lemmatize(raw_word) <span class="hljs-keyword">for</span> raw_word <span class="hljs-keyword">in</span> raw_words]

<span class="hljs-comment"># &#x53BB;&#x9664;&#x505C;&#x7528;&#x8BCD;</span>
filtered_words = [word <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> words <span class="hljs-keyword">if</span> word <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> stopwords.words(<span class="hljs-string">&apos;english&apos;</span>)]

print(<span class="hljs-string">&apos;&#x539F;&#x59CB;&#x6587;&#x672C;&#xFF1A;&apos;</span>, raw_text)
print(<span class="hljs-string">&apos;&#x9884;&#x5904;&#x7406;&#x7ED3;&#x679C;&#xFF1A;&apos;</span>, filtered_words)
</code></pre>
<p>&#x8FD0;&#x884C;&#x7ED3;&#x679C;&#xFF1A;</p>
<pre><code class="lang-python">&#x539F;&#x59CB;&#x6587;&#x672C;&#xFF1A; Life is like a box of chocolates. You never know what you&apos;re gonna get.
&#x9884;&#x5904;&#x7406;&#x7ED3;&#x679C;&#xFF1A; [&apos;Life&apos;, &apos;like&apos;, &apos;box&apos;, &apos;chocolate&apos;, &apos;.&apos;, &apos;You&apos;, &apos;never&apos;, &apos;know&apos;, &quot;&apos;re&quot;, &apos;gon&apos;, &apos;na&apos;, &apos;get&apos;, &apos;.&apos;]
</code></pre>
<h3 id="&#x4F7F;&#x7528;&#x6848;&#x4F8B;&#xFF1A;">&#x4F7F;&#x7528;&#x6848;&#x4F8B;&#xFF1A;</h3>
<pre><code class="lang-python"><span class="hljs-keyword">import</span> nltk
<span class="hljs-keyword">from</span> nltk.tokenize <span class="hljs-keyword">import</span> WordPunctTokenizer

sent_tokenizer = nltk.data.load(<span class="hljs-string">&apos;tokenizers/punkt/english.pickle&apos;</span>)  
paragraph = <span class="hljs-string">&quot;The first time I heard that song was in Hawaii on radio.  I was just a kid, and loved it very much! What a fantastic song!&quot;</span>  

<span class="hljs-comment"># &#x5206;&#x53E5;</span>
sentences = sent_tokenizer.tokenize(paragraph) 
print(sentences)

sentence = <span class="hljs-string">&quot;Are you old enough to remember Michael Jackson attending. the Grammys with Brooke Shields and Webster sat on his lap during the show?&quot;</span>  

<span class="hljs-comment"># &#x5206;&#x8BCD;</span>
words = WordPunctTokenizer().tokenize(sentence.lower())  
print(words)
</code></pre>
<p>&#x8F93;&#x51FA;&#x7ED3;&#x679C;&#xFF1A;</p>
<pre><code class="lang-python">[<span class="hljs-string">&apos;The first time I heard that song was in Hawaii on radio.&apos;</span>, <span class="hljs-string">&apos;I was just a kid, and loved it very much!&apos;</span>, <span class="hljs-string">&apos;What a fantastic song!&apos;</span>]

[<span class="hljs-string">&apos;are&apos;</span>, <span class="hljs-string">&apos;you&apos;</span>, <span class="hljs-string">&apos;old&apos;</span>, <span class="hljs-string">&apos;enough&apos;</span>, <span class="hljs-string">&apos;to&apos;</span>, <span class="hljs-string">&apos;remember&apos;</span>, <span class="hljs-string">&apos;michael&apos;</span>, <span class="hljs-string">&apos;jackson&apos;</span>, <span class="hljs-string">&apos;attending&apos;</span>, <span class="hljs-string">&apos;.&apos;</span>, <span class="hljs-string">&apos;the&apos;</span>, <span class="hljs-string">&apos;grammys&apos;</span>, <span class="hljs-string">&apos;with&apos;</span>, <span class="hljs-string">&apos;brooke&apos;</span>, <span class="hljs-string">&apos;shields&apos;</span>, <span class="hljs-string">&apos;and&apos;</span>, <span class="hljs-string">&apos;webster&apos;</span>, <span class="hljs-string">&apos;sat&apos;</span>, <span class="hljs-string">&apos;on&apos;</span>, <span class="hljs-string">&apos;his&apos;</span>, <span class="hljs-string">&apos;lap&apos;</span>, <span class="hljs-string">&apos;during&apos;</span>, <span class="hljs-string">&apos;the&apos;</span>, <span class="hljs-string">&apos;show&apos;</span>, <span class="hljs-string">&apos;?&apos;</span>]
</code></pre>
<footer class="page-footer"><span class="copyright">Copyright &#xA9; BigCat all right reserved&#xFF0C;powered by Gitbook</span><span class="footer-modification">&#x300C;Revision Time:
2017-04-27 00:21:20&#x300D;
</span></footer>
                    
                    </section>
                
                
                </div>
            </div>
        </div>

        
        <a href="../../file/part06/6.html" class="navigation navigation-prev " aria-label="Previous page: 五、自然语言处理NLTK"><i class="fa fa-angle-left"></i></a>
        
        
        <a href="../../file/part06/6.2.html" class="navigation navigation-next " aria-label="Next page: jieba分词"><i class="fa fa-angle-right"></i></a>
        
    </div>
</div>

        
<script src="../../gitbook/app.js"></script>

    
    <script src="../../gitbook/plugins/gitbook-plugin-splitter/splitter.js"></script>
    

    
    <script src="../../gitbook/plugins/gitbook-plugin-toggle-chapters/toggle.js"></script>
    

    
    <script src="../../gitbook/plugins/gitbook-plugin-fontsettings/buttons.js"></script>
    

    
    <script src="../../gitbook/plugins/gitbook-plugin-livereload/plugin.js"></script>
    

<script>
require(["gitbook"], function(gitbook) {
    var config = {"disqus":{"shortName":"gitbookuse"},"github":{"url":"https://github.com/dododream"},"search-pro":{"cutWordLib":"nodejieba","defineWord":["gitbook-use"]},"sharing":{"weibo":true,"facebook":true,"twitter":true,"google":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"tbfed-pagefooter":{"copyright":"Copyright © BigCat","modify_label":"「Revision Time:","modify_format":"YYYY-MM-DD HH:mm:ss」"},"baidu":{"token":"ff100361cdce95dd4c8fb96b4009f7bc"},"sitemap":{"hostname":"http://www.treenewbee.top"},"donate":{"wechat":"http://weixin.png","alipay":"http://alipay.png","title":"","button":"赏","alipayText":"支付宝打赏","wechatText":"微信打赏"},"edit-link":{"base":"https://github.com/dododream/edit","label":"Edit This Page"},"splitter":{},"toggle-chapters":{},"highlight":{},"fontsettings":{"theme":"white","family":"sans","size":2},"livereload":{}};
    gitbook.start(config);
});
</script>

        <!-- body:end -->
    </body>
    <!-- End of book Python数据分析课程讲义 -->
</html>
